﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using DataProcessor.Core;
using System.Xml.Linq;
using System.Text.RegularExpressions;
using LumenWorks.Framework.IO.Csv;

namespace DataProcessor.DeeplinkData
{
    public class DeeplinkDataProcessor
    {
        public DeeplinkDataProcessor()
        { 
        }

        /// <summary>
        /// Get the number of dublicate records in two different files containing videoIds in each line
        /// </summary>
        /// <param name="file1"></param>
        /// <param name="file2"></param>
        /// <returns></returns>
        public int GetNumVideoIdDuplicates(string file1, string file2)
        {
            return File.ReadAllLines(file2).Join(File.ReadAllLines(file1), v1 => v1, v2 => v2, (v1, v2) => v1).Count();
        }

        /// <summary>
        /// Given the deeplinks with their time code as well as indexes to find MSRA structure file, 
        /// this module will open the structure file for each video and for each shot in video it checks if
        /// the shot has deeplink or not and finally ouput the results to csv
        /// </summary>
        /// <param name="deeplinksPath"></param>
        /// <param name="videoIndicesPath"></param>
        /// <param name="msraDataPath"></param>
        /// <param name="outputPath"></param>
        public void GetDeeplinksPerShot(string deeplinksPath, string videoIndicesPath, string msraDataPath, string outputPath)
        { 
            // 1) Get video structure file location and load the file
            // 2) For each shot in this file, check if the shot has deeplink in deeplink file
            // 3) Write output:  videoId, keyframeId, class (0, 1)

            var paths = File.ReadAllLines(videoIndicesPath).ToCsvDictionary()
                .Select(i => new { VideoId = i["video_id"], Set = i["set"], Query = i["query"], Index = i["index"] })
                .Select(i => new
                {
                    VideoId = i.VideoId,
                    Path = String.Format(@"{0}\Video_{1}\Structure\{2}\{3}",
                        msraDataPath,
                        i.Set + ".0",
                        i.Query,
                        i.Query + "_" + i.Index + ".xml")
                })
                .ToDictionary(i => i.VideoId, i => i.Path);

            // Get all time codes from deeplink file
            var timecodes = File.ReadAllLines(deeplinksPath).ToCsvDictionary()
                .Select(i => new { TimeCode = i["timecode"], VideoId = i["video_id"] });

            Console.WriteLine("Total number of videos to processed: {0}", timecodes.Select(t => t.VideoId).Distinct().Count());

            // 1) Make a disitic list of video ids from deeplinks (timecodes) file
            // 2) Open the structure file for each vide and select all shots
            // 3) Join the above data with timecodes and check for any shot if any of the timecodes lies in each shot
            // 4) Create output and convert it to csv
            var keyframes = timecodes.Select(t => t.VideoId).Distinct()
                .SelectMany((vId, ix) =>
                {
                    Console.WriteLine(ix);
                    return XDocument.Load(paths[vId]).Descendants("Shot")
                        .Select(s => new
                        {
                            VideoId = vId,
                            KeyframeId = s.Descendants("FrameId").FirstOrDefault().Value,
                            ContainDeeplink = timecodes.Where(t => t.VideoId == vId)
                                .Any(t => ContainsTimeCode(
                                    t.TimeCode,
                                    s.Descendants("BgnFrameTime").FirstOrDefault().Value,
                                    s.Descendants("EndFrameTime").FirstOrDefault().Value))
                        });
                })
                .Select(kf => String.Format("{0},{1},{2}", kf.VideoId, kf.KeyframeId, kf.ContainDeeplink ? "1" : "0"));

            var header = new string[] { "VideoId,KeyframeId,Deeplink" };
            File.WriteAllLines(outputPath, header.Concat(keyframes));
        }

        /// <summary>
        /// Create a csv file containing VideoId,ShotBeginTime,ShotEndTime,KeyframeTime
        /// This method instansly write to output file
        /// </summary>
        /// <param name="videoIndicesPath"></param>
        /// <param name="msraDataPath"></param>
        /// <param name="outputPath"></param>
        /// <param name="videIdsPath">Path to csv file containing videoIds to filter based on those Ids</param>
        public void GetVideoShotsAndKeyframes(string videoIndicesPath, string msraDataPath, string outputPath, string videIdsPath)
        {
            var writer = new StreamWriter(File.OpenWrite(outputPath));

            var videos = File.ReadAllLines(videoIndicesPath).ToCsvDictionary()
                .Select(i => new { VideoId = i["video_id"], Set = i["set"], Query = i["query"], Index = i["index"] })
                .Select(i => new
                {
                    VideoId = i.VideoId,
                    Path = String.Format(@"{0}\Video_{1}\Structure\{2}\{3}",
                        msraDataPath,
                        i.Set + ".0",
                        i.Query,
                        i.Query + "_" + i.Index + ".xml")
                });

            writer.WriteLine("VideoId,ShotStart,ShotEnd,Keyframe");
            
            // If videoIds are provided, create the output for only those video Ids
            if (!String.IsNullOrEmpty(videIdsPath))
                videos = videos.Join(File.ReadAllLines(videIdsPath), v => v.VideoId, vId => vId, (v, vId) => v);

            videos.ToList()
            .ForEach(i =>
            {
                XDocument.Load(i.Path).Descendants("Shot").Select(el => String.Format("{0},{1},{2},{3}",
                    i.VideoId,
                    el.Descendants("BgnFrameTime").FirstOrDefault().Value,
                    el.Descendants("EndFrameTime").FirstOrDefault().Value,
                    el.Descendants("BeginTime").FirstOrDefault().Value
                ))
                .ToList()
                .ForEach(line => writer.WriteLine(line));
            });

            writer.Close();
        }

        /// <summary>
        /// Given the index file to find path of MSRA videos, keyframeIds of each video and a feature name,
        /// this method will read the features for each keyframeId and create a feature file in LIBSVM format
        /// The format would be "Deeplink (class) feat1:value1 feat2:value2 .... "
        /// </summary>
        /// <param name="videoIndecesPath"></param>
        /// <param name="keyframesPath"></param>
        /// <param name="msraDataPath"></param>
        /// <param name="outputPath"></param>
        /// <param name="feature"></param>
        /// <param name="skip">Number of videos in keyframes file to skip</param>
        /// <param name="take">Number of videos in keyframes file to take</param>
        public void GetLibSvmFeatures(string videoIndecesPath, string keyframesPath, string msraDataPath, string outputPath, string feature, int skip, int take)
        {
            // Read the videoInices path and conver it to dictionary to find the path of its feature file for a given VideId
            var paths = File.ReadAllLines(videoIndecesPath).ToCsvDictionary()
               .Select(i => new { VideoId = i["video_id"], Set = i["set"], Query = i["query"], Index = i["index"] })
               .Select(i => new
               {
                   VideoId = i.VideoId,
                   Path = String.Format(@"{0}\Video_{1}\KeyframeFeature\{2}\{3}\{4}",
                       msraDataPath,
                       i.Set + ".0",
                       feature,
                       i.Query,
                       i.Query + "_" + i.Index + ".txt")
               })
               .ToDictionary(i => i.VideoId, i => i.Path);

            // Read keyframes
            var keyframes = File.ReadAllLines(keyframesPath).ToCsvDictionary()
                .Select(i => new { VideoId = i["VideoId"], KeyframeId = i["KeyframeId"], Deeplink = i["Deeplink"] });

            // 1) Get distinct videoIds in keyframes file
            var features = keyframes.Select(kf => kf.VideoId).Distinct().Skip(skip).Take(take)
                // 2) Open feature file for each videId and read all KeyframeIds and Features
                .SelectMany(vId => File.ReadAllLines(paths[vId])
                    .Select(line =>
                    {
                        string[] parts = line.Split(new char[] { '\t' }, 2);

                        return new { KeyframeId = parts[0], Features = parts[1], VideoId = vId };
                    }))
                // 3) Join the above data with keyframes based on VideoId and KeyFrameId (to get class label)
                .Join(keyframes, f => new { f.KeyframeId, f.VideoId }, kf => new { kf.KeyframeId, kf.VideoId }, (f, kf) => new 
                    { 
                        KeyframeId = f.KeyframeId, 
                        Deeplink = kf.Deeplink, 
                        Features = f.Features,
                        VideoId = f.VideoId
                    })
                // 4) Spilit features and output it to LIBSVM format
                .Select(i => String.Format("{0} {1}", 
                    i.Deeplink,
                    i.Features.Split(new string[] {"   "}, StringSplitOptions.RemoveEmptyEntries)
                        .Select((feat, ix) => (ix + 1) + ":" + feat).Aggregate((cur, next) => cur + " " + next)));

            File.WriteAllLines(outputPath, features);
        }

        /// <summary>
        /// Given the the index of videos, this module find the phusical location of structure file of each video in MSRA dataset,
        /// read the shot number of each video and output it to csv
        /// </summary>
        /// <param name="videoIndicesPath"></param>
        /// <param name="msraDataPath"></param>
        /// <param name="outputPath"></param>
        public void GetShotNum(string videoIndicesPath, string msraDataPath, string outputPath)
        {
            var shots = File.ReadAllLines(videoIndicesPath).ToCsvDictionary()
                .Select(i => new { VideoId = i["video_id"], Set = i["set"], Query = i["query"], Index = i["index"] })
                .Select(i => new
                {
                    VideoId = i.VideoId,
                    Path = String.Format(@"{0}\Video_{1}\Structure\{2}\{3}",
                        msraDataPath,
                        i.Set + ".0",
                        i.Query,
                        i.Query + "_" + i.Index + ".xml")
                })
                .Select((i, ix) =>
                {
                    // Write down the number of video structure files processed until now
                    Console.WriteLine(ix);

                    // Load structure xml document, read the shotsNum and output it to csv
                    return String.Format("{0},{1}",
                        i.VideoId,
                        XDocument.Load(i.Path).Element("VideoStrcuture").Element("Shots").Attribute("ShotsNum").Value);
                });

            var header = new string[] { "VideoId,ShotsNum" };
            File.WriteAllLines(outputPath, header.Concat(shots));
        }

        /// <summary>
        /// Join the main dataset which is generated from the python script with videoShots and add the shot number of each video in csv
        /// </summary>
        /// <param name="mainFeatureFile">The main csv feature file</param>
        /// <param name="videoShotsFile">The csv file of video shots number</param>
        /// <param name="outputFile">The output csv file</param>
        public void JoinFeatureFiles(string mainFeatureFile, string videoShotsFile, string outputFile)
        {
            // Shotnumbers for each video
            var shots = File.ReadAllLines(videoShotsFile).ToCsvDictionary()
                .Select(i => new { VideoId = i["VideoId"], ShotNum = i["ShotsNum"] });
            
            // Read main feature files which is csv with " as qoute char
            CsvReader csv = new CsvReader(new StreamReader(mainFeatureFile), false);
            
            // Parse csv and join it with shot number based on videoId and again convert it to csv
            var output = csv.Select(fields => new 
                { 
                    VideoId = fields[0],
                    Title = fields[1],
                    Description = fields[2],
                    Category = fields[3],
                    Duration = fields[4],
                    ViewCount = fields[5],
                    NumLike = fields[6],
                    NumDislikes = fields[7],
                    NumComments = fields[8],
                    Deeplink = fields[9]
                })
                .Join(shots, feat => feat.VideoId, sh => sh.VideoId, (feat, sh) => new
                {
                    Feats = feat,
                    sh.ShotNum
                })
                .Select(i => String.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\",\"{7}\",\"{8}\",\"{9}\",\"{10}\"",
                    i.Feats.VideoId,
                    i.Feats.Title.Replace("\"","\"\""),
                    i.Feats.Description.Replace("\"", "\"\""),
                    i.Feats.Category,
                    i.Feats.Duration,
                    i.Feats.ViewCount,
                    i.Feats.NumLike,
                    i.Feats.NumDislikes,
                    i.Feats.NumComments,
                    i.Feats.Deeplink,
                    i.ShotNum));

            File.WriteAllLines(outputFile, output);
        }

        /// <summary>
        /// Split the given video dataset which is in csv format into two separated train and test parts
        /// </summary>
        /// <param name="dataset"></param>
        /// <param name="trainFile"></param>
        /// <param name="testFile"></param>
        /// <param name="trainPerc">Percentage of data to be considered as training set</param>
        public void SplitDataset(string dataset, string trainFile, string testFile, double trainPerc)
        {
           
            var csv = new CsvReader(new StreamReader(dataset), false)
                .Select(fields => fields.Aggregate((cur, next) => cur + ",\"" + next.Replace("\"","\"\"") + "\"")).ToList();

            int total = csv.Count();
            int trainSize = Convert.ToInt32(total * trainPerc);
            int testSize = total - trainSize;

            File.WriteAllLines(trainFile, csv.Take(trainSize));
            File.WriteAllLines(testFile, csv.Skip(trainSize).Take(testSize));
        }



        /// <summary>
        /// Check if a given time code in format dd:dd or ddmdds lies in the interval between begintime and endtiem
        /// </summary>
        /// <param name="timecode"></param>
        /// <param name="beginTime"></param>
        /// <param name="endTime"></param>
        /// <returns></returns>
        private bool ContainsTimeCode(string timecode, string beginTime, string endTime)
        {
            int totalSec = GetTimeCodeInSeconds(timecode);

            if (totalSec != -1)
            { 
                return (totalSec >= long.Parse(beginTime) * Math.Pow(10, -7)) 
                    && (totalSec <= long.Parse(endTime) * Math.Pow(10, -7));
            } 

            return false;
        }

        private int GetTimeCodeInSeconds(string timecode)
        {
            Match m = new Regex("((?<min>\\d+):(?<sec>\\d+))|((?<min>\\d+)m(?<sec>\\d+))s").Match(timecode);

            string min = m.Groups["min"].Value, sec = m.Groups["sec"].Value;

            if (!String.IsNullOrEmpty(min) && !String.IsNullOrEmpty(sec))
            {
                return int.Parse(min) * 60 + int.Parse(sec);
            }

            return -1;
        }

        public List<string> ExtractTimeCodes(string comment)
        {
            MatchCollection m = new Regex(@"(?<!\d|:)((?<timecode1>(\d{1,2}):(\d{2}))|(?<timecode2>(\d{1,2})m(\d{1,2})s))(?!\d|:)").Matches(comment);

            return Enumerable.Range(0, m.Count).Select(i => m[i].Value).ToList();
        }

        private int GetShoutBoundryInSeconds(string value)
        {
            return (int) (long.Parse(value) * Math.Pow(10, -7));
        }

        /// <summary>
        /// Get the aggregated votes of users to each video
        /// </summary>
        /// <param name="amtResult">The original AMT result file</param>
        /// <param name="outputFile"></param>
        public void GetPerVideoVotes(string amtResult, string outputFile)
        {
            var output = new CsvReader(new StreamReader(amtResult), true)
                // Get fields that contains turkers answers
                .Select(fields => new { HitId = fields[0], Q1Deeplink = fields[30], Q2Deeplink = fields[34], Q3Deeplink = fields[38] })
                .GroupBy(i => i.HitId)
                // Transform a single record containing 3 answers to 3 records of one answer
                .SelectMany(g => Enumerable.Range(1, 3).Select(c =>
                    {
                        // Depending the value of counter (c) Q1, Q2 or Q3 would be selected with this delegate
                        var q = new Func<dynamic, string>(d => (c == 1) ? d.Q1Deeplink : (c == 2) ? d.Q2Deeplink : d.Q3Deeplink);
                        return new
                        {
                            HitId = g.Key,
                            StartPoint = g.Where(i => q(i) == "startpoint").Count(),
                            EventOrEnd = g.Where(i => q(i) == "endpoint" || q(i) == "event" || q(i) == "highlight").Count(),
                            //Highlight = g.Where(i => q(i) == "highlight").Count(),
                            Toc = g.Where(i => q(i) == "TOC").Count(),
                            NoTc = g.Where(i => q(i) == "noTC").Count(),
                            Other = g.Where(i => q(i) == "other").Count()
                        };
                    }))
                .Select(i => String.Format("{0},{1},{2},{3},{4},{5}",
                    i.HitId,
                    i.StartPoint,
                    i.EventOrEnd,
                    //i.Highlight,
                    i.Toc,
                    i.NoTc,
                    i.Other));


            var header = new string[] { "HitId,StartPoint,EvenOrEndOrHighlight,Toc,NoTc,Other" };
            File.WriteAllLines(outputFile, header.Concat(output));
        }

        /// <summary>
        /// Filter csv dataset and select a subset of fields specified by features array
        /// </summary>
        /// <param name="dataset"></param>
        /// <param name="outputFile"></param>
        /// <param name="features">Name of the fields that you want to select</param>
        public void GetDataSetFeatures(string dataset, string outputFile, string[] features)
        { 
            var csv = new CsvReader(new StreamReader(dataset), true);
            
            List<string> lines = new List<string>();
            while (csv.ReadNextRecord())
            {
                lines.Add(features.Select(f => csv[f]).Aggregate((cur, next) => cur + "," + next));
            }

            var header = new string[] { String.Join(",", features) };
            File.WriteAllLines(outputFile, header.Concat(lines));
        }

        public void CreateFFMpegCommandForAllVideos(string dirPath, string outputFile)
        {
            var output = Directory.GetFiles(dirPath).Select(f => String.Format("ffmpeg -i \"{0}\" \"{1}\"",
                f,
                (f.Substring(0, f.IndexOf('.')) + ".mp3").Insert(f.LastIndexOf('\\') + 1, "mp3\\")));

            File.WriteAllLines(outputFile, output.Skip(20).Take(40));
        }

        public void CreateFFMpegCommandForAllVideos2(string videoPaths, string outputFile, string savePath = "")
        {
            var output = File.ReadAllLines(videoPaths).Select(f => String.Format("ffmpeg -i \"{0}\" \"{1}\"",
                f,
                String.IsNullOrEmpty(savePath) ? 
                    (f.Substring(0, f.IndexOf('.')) + ".mp3").Insert(f.LastIndexOf('\\') + 1, "mp3\\") : 
                    savePath + "\\" + f.Substring(f.LastIndexOf('\\') + 1, 11) + ".mp3"
                ));

            File.WriteAllLines(outputFile, output);
        }

        public void GenerateMatlabCodeToDrawAudioDeeplinks(string deeplinksPath, string audioDirPath, string shotsPath, string outputPath)
        {
            var mp3Files = Directory.GetFiles(audioDirPath).Where(f => f.Contains(".mp3"))
                .Select(f => new 
                {
                    VideoId = f.Substring(f.LastIndexOf('\\') + 1, f.IndexOf(".mp3") - f.LastIndexOf('\\') - 1),
                    Path = f
                });
            
            //File.WriteAllLines("DeeplinkData/temp.txt", mp3Files.Select(m => m.VideoId));
            
            var shots = File.ReadAllLines(shotsPath).ToCsvDictionary()
                .Select(i => new 
                { 
                    VideoId = i["VideoId"], 
                    ShotStart = GetShoutBoundryInSeconds(i["ShotStart"]),
                    ShotEnd = GetShoutBoundryInSeconds(i["ShotEnd"]),
                    Keyframe = GetShoutBoundryInSeconds(i["Keyframe"])
                });

            string template = "cmpnoveltydeeplink('{0}', [{1}], [{2}], [{3}], [{4}], 5, {5}, 'Video ID: \"{6}\"')";
            Random r = new Random(DateTime.Now.Millisecond);

            var output = File.ReadAllLines(deeplinksPath).ToCsvDictionary()
                .Select(i => new { Deeplink = GetTimeCodeInSeconds(i["TimeCode"]) + r.NextDouble() - 0.5, VideoId = i["VideoId"], Type = i["DeeplinkType"] })
                .Join(mp3Files, o => o.VideoId, i => i.VideoId, (o, i) => new { o.Deeplink, o.VideoId, o.Type, i.Path })
                .GroupBy(i => i.VideoId)
                //.Where(g => g.Count() > 2)
                .Select(g => new
                {
                    VideoId = g.Key,
                    DeeplinksA = g.Where(i => i.Type == "A").Aggregate("", (cur, next) => cur + " " + next.Deeplink),
                    DeeplinksV = g.Where(i => i.Type == "V").Aggregate("", (cur, next) => cur + " " + next.Deeplink),
                    DeeplinksAV = g.Where(i => i.Type == "AV").Aggregate("", (cur, next) => cur + " " + next.Deeplink),
                    Path = g.Select(i => i.Path).FirstOrDefault(),
                    MaxDeeplink = g.Max(i => i.Deeplink),
                    Type = g.Select(i => i.Type).FirstOrDefault()
                })
                .Join(shots, o => o.VideoId, sh => sh.VideoId, (o, sh) => new { o.DeeplinksA, o.DeeplinksV, o.DeeplinksAV, o.VideoId, o.Path, o.MaxDeeplink, o.Type, sh.ShotEnd, sh.Keyframe })
                .GroupBy(i => i.VideoId)
                .Select(g => new
                {
                    VideoId = g.Key,
                    Path = g.Select(i => i.Path).FirstOrDefault(),
                    MaxShotEnd = g.Max(i => i.ShotEnd),
                    DeeplinksA = g.Select(i => i.DeeplinksA).FirstOrDefault(),
                    DeeplinksV = g.Select(i => i.DeeplinksV).FirstOrDefault(),
                    DeeplinksAV = g.Select(i => i.DeeplinksAV).FirstOrDefault(),
                    ShotBoundries = g.Aggregate("", (cur, next) => cur + " " + next.ShotEnd)
                })
                //.Where(i => i.MaxShotEnd < 300)
                .Select((i, ix) => String.Format(template, i.Path, i.DeeplinksA, i.DeeplinksV, i.DeeplinksAV, i.ShotBoundries, ix % 5 + 1, i.VideoId))
                .Select((s, ix) => ix % 5 == 0 ? "figure;\n" + s : s);

            File.WriteAllLines(outputPath, output);
        }

        public void SelectRandomVideoIds(string deeplinksPath, string outputPath, int count)
        {
            var output = File.ReadAllLines(deeplinksPath).ToCsvDictionary()
                .Select(i => i["video_id"])
                .GroupBy(i => i)
                .Select(g => new { VideoId = g.Key, Count = g.Count() })
                .Where(i => i.Count >= 3)
                .Distinct()
                .Take(count)
                .Select(i => i.VideoId);

            File.WriteAllLines(outputPath, output);
        }

        /// <summary>
        /// Find the path of downloaded videos from the server and write the paths to a csv file
        /// </summary>
        /// <param name="videoIdsFile"></param>
        /// <param name="serverPath"></param>
        /// <param name="outputPath"></param>
        public void FindVideoPathsOnServer(string videoIdsFile, string serverPath, string outputPath)
        {
            var videoIds = File.ReadAllLines(videoIdsFile);

            var output = Directory.GetFiles(serverPath, "*.*", SearchOption.AllDirectories)
                .Where(f => videoIds.Any(v => f.Contains(v) && !f.Contains(".mp3"))).Distinct();

            File.WriteAllLines(outputPath, output);
        }

        public void GetHitVideoIds(string deeplinkPath, string outputPath)
        {
            File.WriteAllLines(outputPath, File.ReadAllLines(deeplinkPath).ToCsvDictionary().Select(i => i["video_id"]).Distinct());
        }

        public void GetPathVideoIds(string dirPath, string outputPath)
        {
            var mp3Files = Directory.GetFiles(dirPath).Where(f => f.Contains(".mp3"))
            .Select(f => new
            {
                VideoId = f.Substring(f.LastIndexOf('\\') + 1, f.IndexOf(".mp3") - f.LastIndexOf('\\') - 1),
                Path = f
            });

            File.WriteAllLines(outputPath, mp3Files.Select(i => i.VideoId));
        }

        /// <summary>
        /// Generate Deeplink files together with its types for AMT result file
        /// </summary>
        /// <param name="amtResultFile"></param>
        /// <param name="outputFile"></param>
        public void GenerateDeeplinksWithTypes(string amtResultFile, string outputFile)
        {
            var output = new CsvReader(new StreamReader(amtResultFile), true).ToCsvDictionary()
                .Select(i => new
                {
                    VideoId = i["Input.VIDEO_ID"],
                    CommentId = i["Input.COMMENT_ID"],
                    Types = GetTcTypesDictionary(i["Answer.tcTypes"]),
                    TimeCodes = ExtractTimeCodes(i["Input.COMMENT"]),
                    TcPair = GetPairsDictionary(i["Answer.tcPairs"])
                }).SelectMany(i =>
                    // Filter out the time-codes which are apearing as a pair time-code
                    i.TimeCodes.Where((tc, ix) => !i.TcPair.ContainsValue(ix + 1))
                        .Select((tc, ix) => new
                        {
                            TimeCode = tc,
                            CommentId = i.CommentId,
                            VideoId = i.VideoId,
                            Type = i.Types.ContainsKey(ix + 1) ? i.Types[ix + 1] : "?",
                            Pair = i.TcPair.ContainsKey(ix + 1) ? i.TimeCodes[i.TcPair[ix + 1] - 1] : ""
                        })
                ).Select(i => String.Format("{0},{1},{2},{3},{4}",
                    i.TimeCode,
                    i.VideoId,
                    i.CommentId,
                    i.Type,
                    i.Pair
                ));

            var header = new string[] { "TimeCode,VideoId,CommentId,DeeplinkType,PairTimeCode" };
            File.WriteAllLines(outputFile, header.Union(output));
        }

        private Dictionary<int, int> GetPairsDictionary(string tcPairs)
        {
            if (String.IsNullOrEmpty(tcPairs))
                return new Dictionary<int, int>();

            return tcPairs.Replace("(", "").Replace(")", "").Split(' ')
                .Select(pair => new
                {
                    left = pair.Split(',')[0],
                    right = pair.Split(',')[1]
                }).ToDictionary(i => int.Parse(i.left), i => int.Parse(i.right));
        }

        private Dictionary<int, string> GetTcTypesDictionary(string tcTypes)
        {
            return tcTypes.Split(' ')
                .Select(pair => new
                {
                    ix = pair.Split(':')[0],
                    Type = pair.Split(':')[1]
                }).ToDictionary(i => int.Parse(i.ix), i => i.Type);
        }

        public void GenerateCommentDatasets(string amtResultFile, string trainFile, string testFile)
        {
            var output = new CsvReader(new StreamReader(amtResultFile), true).ToCsvDictionary()
            .Select(i => new
            {
                VideoId = i["Input.VIDEO_ID"],
                Comment = i["Input.COMMENT"],
                Types = i["Answer.tcTypes"]
            })//.Where(i => i.Types == "1:A" || i.Types == "1:V")
            .Select(i => String.Format("\"{0}\",\"{1}\",\"{2}\"", i.VideoId, Regex.Replace(i.Comment, @"<[^>]*>", " ").Replace("\"","\"\""), i.Types)).ToList();

            int trainCount = Convert.ToInt32(output.Count() * 0.7);

            var header = new String[] { "VideoId,Comment,TcType" };
            
            File.WriteAllLines(trainFile, output.Take(trainCount));
            File.WriteAllLines(testFile, output.Skip(trainCount));

            Console.WriteLine(output.Count());
            Console.Read();
        }

        public void GenerateDeeplinkTypeForVideos(string commentDataFile, string outputFile)
        {
            var output = new CsvReader(new StreamReader(commentDataFile), true).ToCsvDictionary()
                .Select(i => new
                {
                    VideoId = i["VideoId"],
                    Comment = i["Comment"],
                    Type = i["TcType"]
                })
                .GroupBy(i => i.VideoId)
                .Select(g => new
                {
                    VideoId = g.Key,
                    TypeA = g.Where(i => i.Type.Contains("1:A")).Count(),
                    TypeV = g.Where(i => i.Type.Contains("1:V")).Count(),
                    TypeAV = g.Where(i => i.Type.Contains("1:AV")).Count(),
                    TypeNoTc = g.Where(i => i.Type.Contains("1:NoTC")).Count(),
                })
                .Select(i => String.Format("{0},{1},{2},{3},{4}", i.VideoId, i.TypeA, i.TypeV, i.TypeAV,i.TypeNoTc));

            var header = new string[] { "VideoId,A,V,AV,NoTC" };
            File.WriteAllLines(outputFile, header.Concat(output));
        }

        public void GetAudioFilesPath(string audioDirPath, string outputFile)
        {
            var mp3Files = Directory.GetFiles(audioDirPath).Where(f => f.Contains(".mp3"))
                .Select(f => new
                {
                    VideoId = f.Substring(f.LastIndexOf('\\') + 1, f.IndexOf(".mp3") - f.LastIndexOf('\\') - 1),
                    Path = f
                });

            File.WriteAllLines(outputFile, mp3Files.Select(m => m.Path));
        }
    }
}
